{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "import pickle\n",
    "import joblib\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "customer_id\n",
       "0          21\n",
       "1          86\n",
       "2          18\n",
       "3           2\n",
       "4          13\n",
       "           ..\n",
       "1371975    51\n",
       "1371976    84\n",
       "1371977    45\n",
       "1371978     7\n",
       "1371979     1\n",
       "Length: 1362281, dtype: int64"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('../Data/transactions_translated.csv',index_col=0)\n",
    "\n",
    "train_set = []\n",
    "valid_set = []\n",
    "cnt = 0\n",
    "a = df.groupby('customer_id').size()\n",
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "23.334630667241193"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8742975935214541"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(a[a<=50])/len(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8352549877741817"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(a[a<=40])/len(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n"
     ]
    }
   ],
   "source": [
    "import random\n",
    "import pickle\n",
    "import joblib\n",
    "import pandas as pd\n",
    "\n",
    "item_feat = {}\n",
    "user_feat = {}\n",
    "MAX_LEN = 50\n",
    "\n",
    "with open('../Data/Processed/customer_feat.pkl', 'rb') as f:\n",
    "    user_feat['customer_id'] = pickle.load(f)\n",
    "    user_feat['FN'] = pickle.load(f)\n",
    "    user_feat['Active'] = pickle.load(f)\n",
    "    user_feat['club_member_status'] = pickle.load(f)\n",
    "    user_feat['fashion_news_frequency'] = pickle.load(f)\n",
    "    user_feat['age'] = pickle.load(f)\n",
    "    customer_cnt = pickle.load(f)\n",
    "    _ = pickle.load(f)\n",
    "\n",
    "with open('../Data/Processed/article_sparse.pkl', 'rb') as f:\n",
    "    item_feat['artid'] = pickle.load(f)\n",
    "    item_feat['pcode'] = pickle.load(f)\n",
    "    _ = pickle.load(f)\n",
    "    _ = pickle.load(f)\n",
    "    artid_cnt, pcode_cnt = pickle.load(f)\n",
    "\n",
    "with open('../Data/Processed/topk_index_100.pkl', 'rb') as f:\n",
    "    topk_index_100 = pickle.load(f)\n",
    "\n",
    "df = pd.read_csv('../Data/transactions_translated.csv',index_col=0)\n",
    "\n",
    "train_set = []\n",
    "valid_set = []\n",
    "cnt = 0\n",
    "for reviewerID, seq in df.groupby('customer_id'):\n",
    "    if cnt % 100000 == 0: print(cnt)\n",
    "    cnt += 1\n",
    "    pos_list = seq['article_id'].tolist()\n",
    "    neg_list = []\n",
    "    for art in pos_list:\n",
    "        neg = pos_list[0]\n",
    "        while (neg in pos_list):\n",
    "            neg = topk_index_100[art-1, random.randint(0,99)]+1\n",
    "        neg_list.append(neg)\n",
    "\n",
    "    for i in range(1, len(pos_list)):\n",
    "        idx = pos_list[max(0,i-MAX_LEN):i]\n",
    "        idx = [x-1 for x in idx]\n",
    "        user_part = tuple(x[reviewerID] for x in user_feat.values())\n",
    "        pos_item_part = tuple(x[pos_list[i]-1] for x in item_feat.values())\n",
    "        neg_item_part = tuple(x[neg_list[i]-1] for x in item_feat.values())\n",
    "        seq_part = tuple(x[idx] for x in item_feat.values())\n",
    "        if i != len(pos_list) - 1:\n",
    "            train_set.append(user_part + pos_item_part + seq_part + (i, 1))\n",
    "            train_set.append(user_part + neg_item_part + seq_part + (i, 0))\n",
    "        else:\n",
    "            valid_set.append(user_part + pos_item_part + seq_part + (i, 1))\n",
    "            valid_set.append(user_part + neg_item_part + seq_part + (i, 0))\n",
    "    break\n",
    "random.shuffle(train_set)\n",
    "random.shuffle(valid_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0,\n",
       "  0,\n",
       "  0,\n",
       "  1,\n",
       "  0,\n",
       "  49.0,\n",
       "  60341,\n",
       "  24830,\n",
       "  array([ 29517,    100,  30328,  50725,  16004,  16004,  23997,  65668,\n",
       "          38173,  79279,  79279,  76504,  49479,  76591,  83623,  90061,\n",
       "          99927, 100485,  78720,  93745]),\n",
       "  array([11547,    37, 11823, 20579,  5599,  5599,  9234, 27086, 15419,\n",
       "         33375, 33375, 32032, 20150, 32092, 35392, 38591, 43671, 43998,\n",
       "         33130, 40534]),\n",
       "  20,\n",
       "  0),\n",
       " (0,\n",
       "  0,\n",
       "  0,\n",
       "  1,\n",
       "  0,\n",
       "  49.0,\n",
       "  16024,\n",
       "  5599,\n",
       "  array([ 29517,    100,  30328,  50725,  16004,  16004,  23997,  65668,\n",
       "          38173,  79279,  79279,  76504,  49479,  76591,  83623,  90061,\n",
       "          99927, 100485,  78720,  93745]),\n",
       "  array([11547,    37, 11823, 20579,  5599,  5599,  9234, 27086, 15419,\n",
       "         33375, 33375, 32032, 20150, 32092, 35392, 38591, 43671, 43998,\n",
       "         33130, 40534]),\n",
       "  20,\n",
       "  1)]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "valid_set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0, 0),\n",
       " (0, 0),\n",
       " (0, 0),\n",
       " (1, 1),\n",
       " (0, 0),\n",
       " (49.0, 49.0),\n",
       " (60341, 16024),\n",
       " (24830, 5599),\n",
       " (array([ 29517,    100,  30328,  50725,  16004,  16004,  23997,  65668,\n",
       "          38173,  79279,  79279,  76504,  49479,  76591,  83623,  90061,\n",
       "          99927, 100485,  78720,  93745]),\n",
       "  array([ 29517,    100,  30328,  50725,  16004,  16004,  23997,  65668,\n",
       "          38173,  79279,  79279,  76504,  49479,  76591,  83623,  90061,\n",
       "          99927, 100485,  78720,  93745])),\n",
       " (array([11547,    37, 11823, 20579,  5599,  5599,  9234, 27086, 15419,\n",
       "         33375, 33375, 32032, 20150, 32092, 35392, 38591, 43671, 43998,\n",
       "         33130, 40534]),\n",
       "  array([11547,    37, 11823, 20579,  5599,  5599,  9234, 27086, 15419,\n",
       "         33375, 33375, 32032, 20150, 32092, 35392, 38591, 43671, 43998,\n",
       "         33130, 40534])),\n",
       " (20, 20),\n",
       " (0, 1)]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = list(zip(*valid_set))\n",
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0),\n",
       " (0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0),\n",
       " (0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0),\n",
       " (1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1),\n",
       " (0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0),\n",
       " (49.0,\n",
       "  49.0,\n",
       "  49.0,\n",
       "  49.0,\n",
       "  49.0,\n",
       "  49.0,\n",
       "  49.0,\n",
       "  49.0,\n",
       "  49.0,\n",
       "  49.0,\n",
       "  49.0,\n",
       "  49.0,\n",
       "  49.0,\n",
       "  49.0,\n",
       "  49.0,\n",
       "  49.0,\n",
       "  49.0,\n",
       "  49.0,\n",
       "  49.0,\n",
       "  49.0,\n",
       "  49.0,\n",
       "  49.0,\n",
       "  49.0,\n",
       "  49.0,\n",
       "  49.0,\n",
       "  49.0,\n",
       "  49.0,\n",
       "  49.0,\n",
       "  49.0,\n",
       "  49.0,\n",
       "  49.0,\n",
       "  49.0,\n",
       "  49.0,\n",
       "  49.0,\n",
       "  49.0,\n",
       "  49.0,\n",
       "  49.0,\n",
       "  49.0),\n",
       " (99927,\n",
       "  100,\n",
       "  79279,\n",
       "  76504,\n",
       "  48801,\n",
       "  16004,\n",
       "  57399,\n",
       "  100485,\n",
       "  82404,\n",
       "  22117,\n",
       "  65668,\n",
       "  101095,\n",
       "  84588,\n",
       "  83623,\n",
       "  79279,\n",
       "  16004,\n",
       "  79690,\n",
       "  54245,\n",
       "  60859,\n",
       "  23997,\n",
       "  69000,\n",
       "  85120,\n",
       "  50725,\n",
       "  54245,\n",
       "  38173,\n",
       "  49479,\n",
       "  77751,\n",
       "  76591,\n",
       "  16023,\n",
       "  9309,\n",
       "  75349,\n",
       "  90061,\n",
       "  78098,\n",
       "  30328,\n",
       "  49470,\n",
       "  93745,\n",
       "  23301,\n",
       "  78720),\n",
       " (43671,\n",
       "  37,\n",
       "  33375,\n",
       "  32032,\n",
       "  19891,\n",
       "  5599,\n",
       "  23455,\n",
       "  43998,\n",
       "  34825,\n",
       "  8325,\n",
       "  27086,\n",
       "  44370,\n",
       "  35806,\n",
       "  35392,\n",
       "  33375,\n",
       "  5599,\n",
       "  33570,\n",
       "  22067,\n",
       "  25029,\n",
       "  9234,\n",
       "  28507,\n",
       "  36038,\n",
       "  20579,\n",
       "  22067,\n",
       "  15419,\n",
       "  20150,\n",
       "  32682,\n",
       "  32092,\n",
       "  5599,\n",
       "  2944,\n",
       "  31511,\n",
       "  38591,\n",
       "  32834,\n",
       "  11823,\n",
       "  20146,\n",
       "  40534,\n",
       "  8881,\n",
       "  33130),\n",
       " (array([29517,   100, 30328, 50725, 16004, 16004, 23997, 65668, 38173,\n",
       "         79279, 79279, 76504, 49479, 76591, 83623, 90061]),\n",
       "  array([29517]),\n",
       "  array([29517,   100, 30328, 50725, 16004, 16004, 23997, 65668, 38173]),\n",
       "  array([29517,   100, 30328, 50725, 16004, 16004, 23997, 65668, 38173,\n",
       "         79279, 79279]),\n",
       "  array([29517,   100, 30328, 50725, 16004, 16004, 23997, 65668, 38173,\n",
       "         79279, 79279, 76504, 49479]),\n",
       "  array([29517,   100, 30328, 50725, 16004]),\n",
       "  array([29517,   100, 30328, 50725, 16004, 16004]),\n",
       "  array([29517,   100, 30328, 50725, 16004, 16004, 23997, 65668, 38173,\n",
       "         79279, 79279, 76504, 49479, 76591, 83623, 90061, 99927]),\n",
       "  array([29517,   100, 30328, 50725, 16004, 16004, 23997]),\n",
       "  array([29517,   100, 30328, 50725, 16004, 16004, 23997, 65668, 38173,\n",
       "         79279, 79279, 76504, 49479, 76591, 83623, 90061]),\n",
       "  array([29517,   100, 30328, 50725, 16004, 16004, 23997]),\n",
       "  array([29517]),\n",
       "  array([29517,   100, 30328, 50725, 16004, 16004, 23997, 65668, 38173,\n",
       "         79279, 79279]),\n",
       "  array([29517,   100, 30328, 50725, 16004, 16004, 23997, 65668, 38173,\n",
       "         79279, 79279, 76504, 49479, 76591]),\n",
       "  array([29517,   100, 30328, 50725, 16004, 16004, 23997, 65668, 38173,\n",
       "         79279]),\n",
       "  array([29517,   100, 30328, 50725]),\n",
       "  array([29517,   100, 30328, 50725, 16004, 16004, 23997, 65668, 38173,\n",
       "         79279, 79279, 76504, 49479, 76591, 83623]),\n",
       "  array([29517,   100, 30328, 50725, 16004, 16004, 23997, 65668, 38173,\n",
       "         79279]),\n",
       "  array([ 29517,    100,  30328,  50725,  16004,  16004,  23997,  65668,\n",
       "          38173,  79279,  79279,  76504,  49479,  76591,  83623,  90061,\n",
       "          99927, 100485,  78720]),\n",
       "  array([29517,   100, 30328, 50725, 16004, 16004]),\n",
       "  array([29517,   100]),\n",
       "  array([ 29517,    100,  30328,  50725,  16004,  16004,  23997,  65668,\n",
       "          38173,  79279,  79279,  76504,  49479,  76591,  83623,  90061,\n",
       "          99927, 100485]),\n",
       "  array([29517,   100, 30328]),\n",
       "  array([29517,   100, 30328, 50725, 16004, 16004, 23997, 65668, 38173]),\n",
       "  array([29517,   100, 30328, 50725, 16004, 16004, 23997, 65668]),\n",
       "  array([29517,   100, 30328, 50725, 16004, 16004, 23997, 65668, 38173,\n",
       "         79279, 79279, 76504]),\n",
       "  array([29517,   100, 30328, 50725, 16004, 16004, 23997, 65668, 38173,\n",
       "         79279, 79279, 76504, 49479, 76591, 83623, 90061, 99927]),\n",
       "  array([29517,   100, 30328, 50725, 16004, 16004, 23997, 65668, 38173,\n",
       "         79279, 79279, 76504, 49479]),\n",
       "  array([29517,   100, 30328, 50725, 16004]),\n",
       "  array([29517,   100, 30328, 50725, 16004, 16004, 23997, 65668, 38173,\n",
       "         79279, 79279, 76504, 49479, 76591]),\n",
       "  array([29517,   100, 30328, 50725]),\n",
       "  array([29517,   100, 30328, 50725, 16004, 16004, 23997, 65668, 38173,\n",
       "         79279, 79279, 76504, 49479, 76591, 83623]),\n",
       "  array([29517,   100, 30328, 50725, 16004, 16004, 23997, 65668]),\n",
       "  array([29517,   100]),\n",
       "  array([29517,   100, 30328, 50725, 16004, 16004, 23997, 65668, 38173,\n",
       "         79279, 79279, 76504]),\n",
       "  array([ 29517,    100,  30328,  50725,  16004,  16004,  23997,  65668,\n",
       "          38173,  79279,  79279,  76504,  49479,  76591,  83623,  90061,\n",
       "          99927, 100485,  78720]),\n",
       "  array([29517,   100, 30328]),\n",
       "  array([ 29517,    100,  30328,  50725,  16004,  16004,  23997,  65668,\n",
       "          38173,  79279,  79279,  76504,  49479,  76591,  83623,  90061,\n",
       "          99927, 100485])),\n",
       " (array([11547,    37, 11823, 20579,  5599,  5599,  9234, 27086, 15419,\n",
       "         33375, 33375, 32032, 20150, 32092, 35392, 38591]),\n",
       "  array([11547]),\n",
       "  array([11547,    37, 11823, 20579,  5599,  5599,  9234, 27086, 15419]),\n",
       "  array([11547,    37, 11823, 20579,  5599,  5599,  9234, 27086, 15419,\n",
       "         33375, 33375]),\n",
       "  array([11547,    37, 11823, 20579,  5599,  5599,  9234, 27086, 15419,\n",
       "         33375, 33375, 32032, 20150]),\n",
       "  array([11547,    37, 11823, 20579,  5599]),\n",
       "  array([11547,    37, 11823, 20579,  5599,  5599]),\n",
       "  array([11547,    37, 11823, 20579,  5599,  5599,  9234, 27086, 15419,\n",
       "         33375, 33375, 32032, 20150, 32092, 35392, 38591, 43671]),\n",
       "  array([11547,    37, 11823, 20579,  5599,  5599,  9234]),\n",
       "  array([11547,    37, 11823, 20579,  5599,  5599,  9234, 27086, 15419,\n",
       "         33375, 33375, 32032, 20150, 32092, 35392, 38591]),\n",
       "  array([11547,    37, 11823, 20579,  5599,  5599,  9234]),\n",
       "  array([11547]),\n",
       "  array([11547,    37, 11823, 20579,  5599,  5599,  9234, 27086, 15419,\n",
       "         33375, 33375]),\n",
       "  array([11547,    37, 11823, 20579,  5599,  5599,  9234, 27086, 15419,\n",
       "         33375, 33375, 32032, 20150, 32092]),\n",
       "  array([11547,    37, 11823, 20579,  5599,  5599,  9234, 27086, 15419,\n",
       "         33375]),\n",
       "  array([11547,    37, 11823, 20579]),\n",
       "  array([11547,    37, 11823, 20579,  5599,  5599,  9234, 27086, 15419,\n",
       "         33375, 33375, 32032, 20150, 32092, 35392]),\n",
       "  array([11547,    37, 11823, 20579,  5599,  5599,  9234, 27086, 15419,\n",
       "         33375]),\n",
       "  array([11547,    37, 11823, 20579,  5599,  5599,  9234, 27086, 15419,\n",
       "         33375, 33375, 32032, 20150, 32092, 35392, 38591, 43671, 43998,\n",
       "         33130]),\n",
       "  array([11547,    37, 11823, 20579,  5599,  5599]),\n",
       "  array([11547,    37]),\n",
       "  array([11547,    37, 11823, 20579,  5599,  5599,  9234, 27086, 15419,\n",
       "         33375, 33375, 32032, 20150, 32092, 35392, 38591, 43671, 43998]),\n",
       "  array([11547,    37, 11823]),\n",
       "  array([11547,    37, 11823, 20579,  5599,  5599,  9234, 27086, 15419]),\n",
       "  array([11547,    37, 11823, 20579,  5599,  5599,  9234, 27086]),\n",
       "  array([11547,    37, 11823, 20579,  5599,  5599,  9234, 27086, 15419,\n",
       "         33375, 33375, 32032]),\n",
       "  array([11547,    37, 11823, 20579,  5599,  5599,  9234, 27086, 15419,\n",
       "         33375, 33375, 32032, 20150, 32092, 35392, 38591, 43671]),\n",
       "  array([11547,    37, 11823, 20579,  5599,  5599,  9234, 27086, 15419,\n",
       "         33375, 33375, 32032, 20150]),\n",
       "  array([11547,    37, 11823, 20579,  5599]),\n",
       "  array([11547,    37, 11823, 20579,  5599,  5599,  9234, 27086, 15419,\n",
       "         33375, 33375, 32032, 20150, 32092]),\n",
       "  array([11547,    37, 11823, 20579]),\n",
       "  array([11547,    37, 11823, 20579,  5599,  5599,  9234, 27086, 15419,\n",
       "         33375, 33375, 32032, 20150, 32092, 35392]),\n",
       "  array([11547,    37, 11823, 20579,  5599,  5599,  9234, 27086]),\n",
       "  array([11547,    37]),\n",
       "  array([11547,    37, 11823, 20579,  5599,  5599,  9234, 27086, 15419,\n",
       "         33375, 33375, 32032]),\n",
       "  array([11547,    37, 11823, 20579,  5599,  5599,  9234, 27086, 15419,\n",
       "         33375, 33375, 32032, 20150, 32092, 35392, 38591, 43671, 43998,\n",
       "         33130]),\n",
       "  array([11547,    37, 11823]),\n",
       "  array([11547,    37, 11823, 20579,  5599,  5599,  9234, 27086, 15419,\n",
       "         33375, 33375, 32032, 20150, 32092, 35392, 38591, 43671, 43998])),\n",
       " (16,\n",
       "  1,\n",
       "  9,\n",
       "  11,\n",
       "  13,\n",
       "  5,\n",
       "  6,\n",
       "  17,\n",
       "  7,\n",
       "  16,\n",
       "  7,\n",
       "  1,\n",
       "  11,\n",
       "  14,\n",
       "  10,\n",
       "  4,\n",
       "  15,\n",
       "  10,\n",
       "  19,\n",
       "  6,\n",
       "  2,\n",
       "  18,\n",
       "  3,\n",
       "  9,\n",
       "  8,\n",
       "  12,\n",
       "  17,\n",
       "  13,\n",
       "  5,\n",
       "  14,\n",
       "  4,\n",
       "  15,\n",
       "  8,\n",
       "  2,\n",
       "  12,\n",
       "  19,\n",
       "  3,\n",
       "  18),\n",
       " (1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  0,\n",
       "  1,\n",
       "  0,\n",
       "  1,\n",
       "  0,\n",
       "  0,\n",
       "  1,\n",
       "  0,\n",
       "  0,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  1,\n",
       "  0,\n",
       "  0,\n",
       "  1,\n",
       "  0,\n",
       "  1,\n",
       "  1,\n",
       "  0,\n",
       "  1,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  1,\n",
       "  0,\n",
       "  1,\n",
       "  0,\n",
       "  1,\n",
       "  0,\n",
       "  1)]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b = list(zip(*train_set))\n",
    "b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2, 38)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(valid_set), len(train_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "38"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(b[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import torch\n",
    "from torch.nn.utils.rnn import pad_sequence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dict, valid_dict = {}, {}\n",
    "keys = list(user_feat.keys()) + list(item_feat.keys()) + ['hist_'+ x for x in item_feat.keys()] + ['seq_length', 'y']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "for key, obj in zip(keys, a):\n",
    "    if key.startswith('hist_'):\n",
    "        valid_dict[key] = pad_sequence([torch.from_numpy(x) for x in obj], batch_first=True).numpy()\n",
    "    else:\n",
    "        valid_dict[key] = np.asarray(obj)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 29517,    100,  30328,  50725,  16004,  16004,  23997,  65668,\n",
       "         38173,  79279,  79279,  76504,  49479,  76591,  83623,  90061,\n",
       "         99927, 100485,  78720,  93745],\n",
       "       [ 29517,    100,  30328,  50725,  16004,  16004,  23997,  65668,\n",
       "         38173,  79279,  79279,  76504,  49479,  76591,  83623,  90061,\n",
       "         99927, 100485,  78720,  93745]])"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "valid_dict['hist_artid']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "customer_id\n",
      "int64\n",
      "FN\n",
      "int64\n",
      "Active\n",
      "int64\n",
      "club_member_status\n",
      "int64\n",
      "fashion_news_frequency\n",
      "int64\n",
      "age\n",
      "float64\n",
      "artid\n",
      "int64\n",
      "pcode\n",
      "int64\n",
      "hist_artid\n",
      "int64\n",
      "hist_pcode\n",
      "int64\n",
      "seq_length\n",
      "int64\n",
      "y\n",
      "int64\n"
     ]
    }
   ],
   "source": [
    "for key, val in valid_dict.items():\n",
    "    print(key)\n",
    "    print(val.dtype)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[tensor([ 29517,    100,  30328,  50725,  16004,  16004,  23997,  65668,  38173,\n",
       "          79279,  79279,  76504,  49479,  76591,  83623,  90061,  99927, 100485,\n",
       "          78720,  93745]),\n",
       " tensor([ 29517,    100,  30328,  50725,  16004,  16004,  23997,  65668,  38173,\n",
       "          79279,  79279,  76504,  49479,  76591,  83623,  90061,  99927, 100485,\n",
       "          78720,  93745])]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[torch.from_numpy(x) for x in obj]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "for key, obj in zip(keys, b):\n",
    "    if key.startswith('hist_'):\n",
    "        train_dict[key] = pad_sequence([torch.from_numpy(x) for x in obj], batch_first=True).numpy()\n",
    "    else:\n",
    "        train_dict[key] = np.asarray(obj)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 29517,    100,  30328,  50725,  16004,  16004,  23997,  65668,\n",
       "         38173,  79279,  79279,  76504,  49479,  76591,  83623,  90061,\n",
       "             0,      0,      0],\n",
       "       [ 29517,      0,      0,      0,      0,      0,      0,      0,\n",
       "             0,      0,      0,      0,      0,      0,      0,      0,\n",
       "             0,      0,      0],\n",
       "       [ 29517,    100,  30328,  50725,  16004,  16004,  23997,  65668,\n",
       "         38173,      0,      0,      0,      0,      0,      0,      0,\n",
       "             0,      0,      0],\n",
       "       [ 29517,    100,  30328,  50725,  16004,  16004,  23997,  65668,\n",
       "         38173,  79279,  79279,      0,      0,      0,      0,      0,\n",
       "             0,      0,      0],\n",
       "       [ 29517,    100,  30328,  50725,  16004,  16004,  23997,  65668,\n",
       "         38173,  79279,  79279,  76504,  49479,      0,      0,      0,\n",
       "             0,      0,      0],\n",
       "       [ 29517,    100,  30328,  50725,  16004,      0,      0,      0,\n",
       "             0,      0,      0,      0,      0,      0,      0,      0,\n",
       "             0,      0,      0],\n",
       "       [ 29517,    100,  30328,  50725,  16004,  16004,      0,      0,\n",
       "             0,      0,      0,      0,      0,      0,      0,      0,\n",
       "             0,      0,      0],\n",
       "       [ 29517,    100,  30328,  50725,  16004,  16004,  23997,  65668,\n",
       "         38173,  79279,  79279,  76504,  49479,  76591,  83623,  90061,\n",
       "         99927,      0,      0],\n",
       "       [ 29517,    100,  30328,  50725,  16004,  16004,  23997,      0,\n",
       "             0,      0,      0,      0,      0,      0,      0,      0,\n",
       "             0,      0,      0],\n",
       "       [ 29517,    100,  30328,  50725,  16004,  16004,  23997,  65668,\n",
       "         38173,  79279,  79279,  76504,  49479,  76591,  83623,  90061,\n",
       "             0,      0,      0],\n",
       "       [ 29517,    100,  30328,  50725,  16004,  16004,  23997,      0,\n",
       "             0,      0,      0,      0,      0,      0,      0,      0,\n",
       "             0,      0,      0],\n",
       "       [ 29517,      0,      0,      0,      0,      0,      0,      0,\n",
       "             0,      0,      0,      0,      0,      0,      0,      0,\n",
       "             0,      0,      0],\n",
       "       [ 29517,    100,  30328,  50725,  16004,  16004,  23997,  65668,\n",
       "         38173,  79279,  79279,      0,      0,      0,      0,      0,\n",
       "             0,      0,      0],\n",
       "       [ 29517,    100,  30328,  50725,  16004,  16004,  23997,  65668,\n",
       "         38173,  79279,  79279,  76504,  49479,  76591,      0,      0,\n",
       "             0,      0,      0],\n",
       "       [ 29517,    100,  30328,  50725,  16004,  16004,  23997,  65668,\n",
       "         38173,  79279,      0,      0,      0,      0,      0,      0,\n",
       "             0,      0,      0],\n",
       "       [ 29517,    100,  30328,  50725,      0,      0,      0,      0,\n",
       "             0,      0,      0,      0,      0,      0,      0,      0,\n",
       "             0,      0,      0],\n",
       "       [ 29517,    100,  30328,  50725,  16004,  16004,  23997,  65668,\n",
       "         38173,  79279,  79279,  76504,  49479,  76591,  83623,      0,\n",
       "             0,      0,      0],\n",
       "       [ 29517,    100,  30328,  50725,  16004,  16004,  23997,  65668,\n",
       "         38173,  79279,      0,      0,      0,      0,      0,      0,\n",
       "             0,      0,      0],\n",
       "       [ 29517,    100,  30328,  50725,  16004,  16004,  23997,  65668,\n",
       "         38173,  79279,  79279,  76504,  49479,  76591,  83623,  90061,\n",
       "         99927, 100485,  78720],\n",
       "       [ 29517,    100,  30328,  50725,  16004,  16004,      0,      0,\n",
       "             0,      0,      0,      0,      0,      0,      0,      0,\n",
       "             0,      0,      0],\n",
       "       [ 29517,    100,      0,      0,      0,      0,      0,      0,\n",
       "             0,      0,      0,      0,      0,      0,      0,      0,\n",
       "             0,      0,      0],\n",
       "       [ 29517,    100,  30328,  50725,  16004,  16004,  23997,  65668,\n",
       "         38173,  79279,  79279,  76504,  49479,  76591,  83623,  90061,\n",
       "         99927, 100485,      0],\n",
       "       [ 29517,    100,  30328,      0,      0,      0,      0,      0,\n",
       "             0,      0,      0,      0,      0,      0,      0,      0,\n",
       "             0,      0,      0],\n",
       "       [ 29517,    100,  30328,  50725,  16004,  16004,  23997,  65668,\n",
       "         38173,      0,      0,      0,      0,      0,      0,      0,\n",
       "             0,      0,      0],\n",
       "       [ 29517,    100,  30328,  50725,  16004,  16004,  23997,  65668,\n",
       "             0,      0,      0,      0,      0,      0,      0,      0,\n",
       "             0,      0,      0],\n",
       "       [ 29517,    100,  30328,  50725,  16004,  16004,  23997,  65668,\n",
       "         38173,  79279,  79279,  76504,      0,      0,      0,      0,\n",
       "             0,      0,      0],\n",
       "       [ 29517,    100,  30328,  50725,  16004,  16004,  23997,  65668,\n",
       "         38173,  79279,  79279,  76504,  49479,  76591,  83623,  90061,\n",
       "         99927,      0,      0],\n",
       "       [ 29517,    100,  30328,  50725,  16004,  16004,  23997,  65668,\n",
       "         38173,  79279,  79279,  76504,  49479,      0,      0,      0,\n",
       "             0,      0,      0],\n",
       "       [ 29517,    100,  30328,  50725,  16004,      0,      0,      0,\n",
       "             0,      0,      0,      0,      0,      0,      0,      0,\n",
       "             0,      0,      0],\n",
       "       [ 29517,    100,  30328,  50725,  16004,  16004,  23997,  65668,\n",
       "         38173,  79279,  79279,  76504,  49479,  76591,      0,      0,\n",
       "             0,      0,      0],\n",
       "       [ 29517,    100,  30328,  50725,      0,      0,      0,      0,\n",
       "             0,      0,      0,      0,      0,      0,      0,      0,\n",
       "             0,      0,      0],\n",
       "       [ 29517,    100,  30328,  50725,  16004,  16004,  23997,  65668,\n",
       "         38173,  79279,  79279,  76504,  49479,  76591,  83623,      0,\n",
       "             0,      0,      0],\n",
       "       [ 29517,    100,  30328,  50725,  16004,  16004,  23997,  65668,\n",
       "             0,      0,      0,      0,      0,      0,      0,      0,\n",
       "             0,      0,      0],\n",
       "       [ 29517,    100,      0,      0,      0,      0,      0,      0,\n",
       "             0,      0,      0,      0,      0,      0,      0,      0,\n",
       "             0,      0,      0],\n",
       "       [ 29517,    100,  30328,  50725,  16004,  16004,  23997,  65668,\n",
       "         38173,  79279,  79279,  76504,      0,      0,      0,      0,\n",
       "             0,      0,      0],\n",
       "       [ 29517,    100,  30328,  50725,  16004,  16004,  23997,  65668,\n",
       "         38173,  79279,  79279,  76504,  49479,  76591,  83623,  90061,\n",
       "         99927, 100485,  78720],\n",
       "       [ 29517,    100,  30328,      0,      0,      0,      0,      0,\n",
       "             0,      0,      0,      0,      0,      0,      0,      0,\n",
       "             0,      0,      0],\n",
       "       [ 29517,    100,  30328,  50725,  16004,  16004,  23997,  65668,\n",
       "         38173,  79279,  79279,  76504,  49479,  76591,  83623,  90061,\n",
       "         99927, 100485,      0]])"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_dict['hist_artid']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "customer_id\n",
      "int64\n",
      "FN\n",
      "int64\n",
      "Active\n",
      "int64\n",
      "club_member_status\n",
      "int64\n",
      "fashion_news_frequency\n",
      "int64\n",
      "age\n",
      "float64\n",
      "artid\n",
      "int64\n",
      "pcode\n",
      "int64\n",
      "hist_artid\n",
      "int64\n",
      "hist_pcode\n",
      "int64\n",
      "seq_length\n",
      "int64\n",
      "y\n",
      "int64\n"
     ]
    }
   ],
   "source": [
    "for key, val in valid_dict.items():\n",
    "    print(key)\n",
    "    print(val.dtype)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'customer_id': array([723, 842, 735, ..., 947, 743,  94]),\n",
       " 'FN': array([0, 1, 0, ..., 0, 0, 0]),\n",
       " 'Active': array([0, 1, 0, ..., 0, 0, 0]),\n",
       " 'club_member_status': array([1, 1, 1, ..., 1, 1, 1]),\n",
       " 'fashion_news_frequency': array([0, 1, 0, ..., 0, 0, 0]),\n",
       " 'age': array([36., 55., 62., ..., 46., 48., 49.]),\n",
       " 'artid': array([  9993,  38606,  88287, ...,  93376,  58377, 104825]),\n",
       " 'pcode': array([ 3231, 15617, 37626, ..., 40350, 23878, 46680]),\n",
       " 'hist_artid': array([[  452,    68, 43600, ...,     0,     0,     0],\n",
       "        [17338, 47335, 16350, ...,     0,     0,     0],\n",
       "        [22940, 50629, 57693, ..., 12599, 88274, 87032],\n",
       "        ...,\n",
       "        [37036, 37312, 37311, ...,     0,     0,     0],\n",
       "        [10682, 10682, 10682, ...,     0,     0,     0],\n",
       "        [13286, 43443, 53485, ...,     0,     0,     0]]),\n",
       " 'hist_pcode': array([[  137,    28, 17931, ...,     0,     0,     0],\n",
       "        [ 6072, 19323,  5731, ...,     0,     0,     0],\n",
       "        [ 8694, 20557, 23594, ...,  4316, 37620, 36987],\n",
       "        ...,\n",
       "        [14858, 14986, 14986, ...,     0,     0,     0],\n",
       "        [ 3518,  3518,  3518, ...,     0,     0,     0],\n",
       "        [ 4556, 17834, 21743, ...,     0,     0,     0]]),\n",
       " 'seq_length': array([ 9, 39, 50, ..., 43, 34,  3]),\n",
       " 'y': array([1, 0, 1, ..., 1, 1, 0])}"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = joblib.load(\"/home2/Dlib_RecSys/SHCF/HM/Data/Processed/train_dict.db\")\n",
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'customer_id': array([623,  60, 883, ..., 792, 637, 414]),\n",
       " 'FN': array([0, 0, 0, ..., 1, 1, 0]),\n",
       " 'Active': array([0, 0, 0, ..., 1, 1, 0]),\n",
       " 'club_member_status': array([1, 1, 1, ..., 1, 1, 1]),\n",
       " 'fashion_news_frequency': array([0, 0, 0, ..., 1, 1, 0]),\n",
       " 'age': array([25., 51., 25., ..., 52., 73., 50.]),\n",
       " 'artid': array([82320, 13899, 84634, ...,  2265, 40288, 43963]),\n",
       " 'pcode': array([34780,  4795, 35831, ...,   550, 16417, 18062]),\n",
       " 'hist_artid': array([[10864, 73046, 73046, ...,     0,     0,     0],\n",
       "        [ 1714,     0,     0, ...,     0,     0,     0],\n",
       "        [47701, 12586, 46881, ...,     0,     0,     0],\n",
       "        ...,\n",
       "        [ 7512,  3708, 17098, ...,     0,     0,     0],\n",
       "        [40292, 45911,  2256, ...,     0,     0,     0],\n",
       "        [51639, 15335, 15334, ...,     0,     0,     0]]),\n",
       " 'hist_pcode': array([[ 3582, 30346, 30346, ...,     0,     0,     0],\n",
       "        [  448,     0,     0, ...,     0,     0,     0],\n",
       "        [19431,  4316, 19148, ...,     0,     0,     0],\n",
       "        ...,\n",
       "        [ 2169,   922,  5986, ...,     0,     0,     0],\n",
       "        [16417, 18812,   550, ...,     0,     0,     0],\n",
       "        [20963,  5361,  5361, ...,     0,     0,     0]]),\n",
       " 'seq_length': array([14,  1, 31, ..., 47, 38, 10]),\n",
       " 'y': array([1, 0, 1, ..., 0, 0, 1])}"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b = joblib.load(\"/home2/Dlib_RecSys/SHCF/HM/Data/Processed/valid_dict.db\")\n",
    "b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([     1,      2,      3, ..., 105540, 105541, 105542]), (105542,))"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "item_feat['artid'], item_feat['pcode'].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(38536,)\n",
      "(38536,)\n",
      "(38536,)\n",
      "(38536,)\n",
      "(38536,)\n",
      "(38536,)\n",
      "(38536,)\n",
      "(38536,)\n",
      "(38536, 50)\n",
      "(38536, 50)\n",
      "(38536,)\n",
      "(38536,)\n"
     ]
    }
   ],
   "source": [
    "for x in a.values():\n",
    "    print(x.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1790,)\n",
      "(1790,)\n",
      "(1790,)\n",
      "(1790,)\n",
      "(1790,)\n",
      "(1790,)\n",
      "(1790,)\n",
      "(1790,)\n",
      "(1790, 50)\n",
      "(1790, 50)\n",
      "(1790,)\n",
      "(1790,)\n"
     ]
    }
   ],
   "source": [
    "for x in b.values():\n",
    "    print(x.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([      0,       1,       2, ..., 1371977, 1371978, 1371979])"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "user_feat['customer_id']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "7c2e65e64076883662e9fbb467097aa6d81839e6b77012bdd0b6d8c4fbfb9623"
  },
  "kernelspec": {
   "display_name": "Python 3.7.11 ('SR-GNN')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.11"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
